import os
import time
import datetime
import ConfigParser

from invenio.config import CFG_SITE_LANG, CFG_ETCDIR, CFG_VERSION
from invenio.bibtask import write_message

from invenio.dbquery import run_sql, serialize_via_marshal, deserialize_via_marshal
from invenio.bibrank_d_rank import get_all_bibrec_scores_fromDB
from invenio.bibrank_d_rank import get_id_rnkMETHOD
from invenio.bibrank_d_rank import merge_2dictionaries
from invenio.bibrank_d_rank import separe_into_2dictionaries
from invenio.bibrank_d_rank import into_db
from invenio.bibrank_d_rank import create_LUT

from invenio.bibrank_d_rank import create_LUT_from_file
from invenio.bibrank_d_rank import normalize_using_LUT
from invenio.bibrank_d_rank import normalize_db
from invenio.bibrank_d_rank import normalize_data_from_file
from invenio.bibrank_d_rank import show_compressed_data
from invenio.bibrank_d_rank import get_bibrec_dict_with_method_score_from_file
from invenio.bibrank_d_rank import create_charts
from invenio.bibrank_d_rank import create_inverse_LUT
from invenio.bibrank_d_rank import split_data_dict
from invenio.bibrank_d_rank import calculate_epsilon
from invenio.bibrank_d_rank import rescaling
from invenio.bibrank_d_rank import detect_optimal_size_appro_LUT
from invenio.bibrank_d_rank import create_value_frequence_table
from invenio.bibrank_d_rank import dict_2_value_frequence_table


data_from_DB = 0
if (data_from_DB):
    try:
        print "CFG_ETCDIR : " + str(CFG_ETCDIR)
        file = CFG_ETCDIR + "/bibrank/d_rank.cfg"
        config = ConfigParser.ConfigParser()
        config.readfp(open(file))
    except StandardError, e:
        pass

    methods = {}

    sql_string = "SELECT name from rnkMETHOD"
    bibrank_meths = run_sql(sql_string)
    for (rank_method_code,) in bibrank_meths:
        print "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
        print "rank_method_code = " + str(rank_method_code)

        cfg_function = config.get("rank_method", "function")
        if config.has_section(cfg_function):
            methods = {}
            methods["function"] = cfg_function
            print "methods = " + str(methods["function"])
            nb_elements_LUT = int(config.get(cfg_function, "nb_elements_LUT"))
            print "nb_elements_LUT = " + str(nb_elements_LUT)

            if config.has_option(cfg_function, str(rank_method_code)) and\
            config.getboolean(cfg_function, str(rank_method_code)):

                #sql_string = "SELECT id FROM rnkMETHOD WHERE name = '%s'" %rank_method_code
                id_rnkMETHOD = run_sql("SELECT id FROM rnkMETHOD WHERE name = %s", (rank_method_code,))

                print "id_rnkMETHOD : " + str(id_rnkMETHOD[0][0])

                lut, reduced_lut = create_LUT(rank_method_code, nb_elements_LUT, 1)
                #reduced_lut = create_reduced_LUT(rank_method_code, nb_elements_LUT)
                print "LUT :"
                print lut
                print "reduced LUT :"
                print reduced_lut
#                create_charts(lut)

else:
    # The percentage in order to split data into two subsets : test set and training set.
    test_percentage = 0.98

    # The default number of Approximated_LUT
    nb_elements_LUT = 11

    # The minimum and maximum size of Approximated_LUT to draw on the chart
    appro_lut_size_min = 2
    appro_lut_size_max = 100

    # Threshold of epsilon, if the difference between two sides of this point is smaller than
    # this threshold -> break et return this point
    delta_epsilon = 0.02

    # The parameters of data processing
    attribute_file_name = "008.dat"
    level_dir = "recalls/"
    depth = 5

    #================== Draw LUT ====================================
    value_frequence_table, data_dict = create_value_frequence_table(attribute_file_name, level_dir, depth)

    # Create LUT, reduced_LUT and draw them
    lut, appro_lut = create_LUT_from_file(value_frequence_table,"d_rank", nb_elements_LUT, 0)
    lut, draw_lut = create_LUT_from_file(value_frequence_table, "d_rank", appro_lut_size_max, 0)

    draw_lut = rescaling(draw_lut)
    appro_lut = rescaling(appro_lut)
    create_charts(draw_lut, appro_lut,"LUT", "Approximated LUT", "Lookup Table")


    # =================Find optimal size of Approximated table ====================
    # loop with diffrent sizes
    epsilon_dict = {}

    # split into two sets : test and trainning
    test_dict, training_dict = split_data_dict(data_dict, test_percentage)
    print "training_dict"
    print training_dict
    for appro_lut_size in range(appro_lut_size_min, appro_lut_size_max):

        # Create test_LUT from test_dict
        value_frequence_test_dict = dict_2_value_frequence_table(test_dict)
        tmp, test_LUT = create_LUT_from_file(value_frequence_test_dict, "test", appro_lut_size, 0)

        # Apply with two LUT
        epsilon = calculate_epsilon(lut, test_LUT, training_dict)

        epsilon_dict[appro_lut_size] = epsilon

    print "epsilon_dict : "
    for key in sorted(epsilon_dict.keys()):
        print str(key) + " ---> " + str(epsilon_dict[key])



    optimal_size = float(detect_optimal_size_appro_LUT(epsilon_dict, delta_epsilon))
    optimal_size_tmp = optimal_size + 0.0001

    print "optimal_size : "
    print optimal_size

    appro_lut_size_dict = {optimal_size:0.0, optimal_size_tmp:epsilon_dict[2]}

    label = "Optimal size = " + str(int(optimal_size))
    create_charts(epsilon_dict, appro_lut_size_dict,"errors in unit = 10^(-2)", label, "Distribution of errors")


    print "C'est fini !!!!!"
